{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 25969,
     "status": "ok",
     "timestamp": 1592668122578,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "x5fdfF6_afzR",
    "outputId": "95adbef1-faf7-4cc6-ce71-4e43e372ee73"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /gdrive\n",
      "/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/gdrive')\n",
    "%cd /gdrive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6439,
     "status": "ok",
     "timestamp": 1592668205609,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "u4uihEVWaoqy",
    "outputId": "70040814-c8d7-4841-a02a-53e832803037"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "/gdrive/My Drive/Colab Notebooks/nlp_task/kenlm\n",
      "/gdrive/My Drive/Colab Notebooks/nlp_task/kenlm\n",
      "corpus_task  insurance_train.txt  train_lm.ipynb  train.txt.gz\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "% cd /gdrive/My\\ Drive/Colab\\ Notebooks/nlp_task/kenlm\n",
    "! pwd\n",
    "! ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1650,
     "status": "ok",
     "timestamp": 1592668184573,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "DPjRCH_aavHc"
   },
   "outputs": [],
   "source": [
    "# import gzip, os, shutil\n",
    "\n",
    "# with gzip.GzipFile('train.txt.gz') as zipf:\n",
    "#     open('insurance_train.txt', \"wb+\").write(zipf.read())     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8568,
     "status": "ok",
     "timestamp": 1592668266080,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "uW2k42VUa9k7"
   },
   "outputs": [],
   "source": [
    "# import requests\n",
    "\n",
    "# request_2 = requests.get(\"https://github.com/Computing-Intelligence/datasource/raw/master/movie_comments.csv\")\n",
    "# with open('movie_comments.csv', 'wb') as f:\n",
    "#     f.write(request_2.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 176555,
     "status": "ok",
     "timestamp": 1592668495122,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "YEqzNRm_bPyh",
    "outputId": "2eb92634-ebf8-47dd-87ad-6f2174719ae8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "libboost-all-dev is already the newest version (1.65.1.0ubuntu1).\n",
      "The following package was automatically installed and is no longer required:\n",
      "  libnvidia-common-440\n",
      "Use 'apt autoremove' to remove it.\n",
      "0 upgraded, 0 newly installed, 0 to remove and 59 not upgraded.\n",
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "libbz2-dev is already the newest version (1.0.6-8.1ubuntu0.2).\n",
      "libbz2-dev set to manually installed.\n",
      "The following package was automatically installed and is no longer required:\n",
      "  libnvidia-common-440\n",
      "Use 'apt autoremove' to remove it.\n",
      "0 upgraded, 0 newly installed, 0 to remove and 59 not upgraded.\n",
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "The following package was automatically installed and is no longer required:\n",
      "  libnvidia-common-440\n",
      "Use 'apt autoremove' to remove it.\n",
      "Suggested packages:\n",
      "  libeigen3-doc libmrpt-dev\n",
      "The following NEW packages will be installed:\n",
      "  libeigen3-dev\n",
      "0 upgraded, 1 newly installed, 0 to remove and 59 not upgraded.\n",
      "Need to get 810 kB of archives.\n",
      "After this operation, 7,128 kB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libeigen3-dev all 3.3.4-4 [810 kB]\n",
      "Fetched 810 kB in 2s (381 kB/s)\n",
      "Selecting previously unselected package libeigen3-dev.\n",
      "(Reading database ... 144328 files and directories currently installed.)\n",
      "Preparing to unpack .../libeigen3-dev_3.3.4-4_all.deb ...\n",
      "Unpacking libeigen3-dev (3.3.4-4) ...\n",
      "Setting up libeigen3-dev (3.3.4-4) ...\n",
      "--2020-06-20 15:52:20--  https://kheafield.com/code/kenlm.tar.gz\n",
      "Resolving kheafield.com (kheafield.com)... 35.196.63.85\n",
      "Connecting to kheafield.com (kheafield.com)|35.196.63.85|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 490441 (479K) [application/x-gzip]\n",
      "Saving to: ‘STDOUT’\n",
      "\n",
      "-                   100%[===================>] 478.95K   335KB/s    in 1.4s    \n",
      "\n",
      "2020-06-20 15:52:22 (335 KB/s) - written to stdout [490441/490441]\n",
      "\n",
      "-- The C compiler identification is GNU 7.5.0\n",
      "-- The CXX compiler identification is GNU 7.5.0\n",
      "-- Check for working C compiler: /usr/bin/cc\n",
      "-- Check for working C compiler: /usr/bin/cc -- works\n",
      "-- Detecting C compiler ABI info\n",
      "-- Detecting C compiler ABI info - done\n",
      "-- Detecting C compile features\n",
      "-- Detecting C compile features - done\n",
      "-- Check for working CXX compiler: /usr/bin/c++\n",
      "-- Check for working CXX compiler: /usr/bin/c++ -- works\n",
      "-- Detecting CXX compiler ABI info\n",
      "-- Detecting CXX compiler ABI info - done\n",
      "-- Detecting CXX compile features\n",
      "-- Detecting CXX compile features - done\n",
      "-- Looking for pthread.h\n",
      "-- Looking for pthread.h - found\n",
      "-- Looking for pthread_create\n",
      "-- Looking for pthread_create - not found\n",
      "-- Looking for pthread_create in pthreads\n",
      "-- Looking for pthread_create in pthreads - not found\n",
      "-- Looking for pthread_create in pthread\n",
      "-- Looking for pthread_create in pthread - found\n",
      "-- Found Threads: TRUE  \n",
      "-- Boost version: 1.65.1\n",
      "-- Found the following Boost libraries:\n",
      "--   program_options\n",
      "--   system\n",
      "--   thread\n",
      "--   unit_test_framework\n",
      "--   chrono\n",
      "--   date_time\n",
      "--   atomic\n",
      "-- Found ZLIB: /usr/lib/x86_64-linux-gnu/libz.so (found version \"1.2.11\") \n",
      "-- Found BZip2: /usr/lib/x86_64-linux-gnu/libbz2.so (found version \"1.0.6\") \n",
      "-- Looking for BZ2_bzCompressInit\n",
      "-- Looking for BZ2_bzCompressInit - found\n",
      "-- Looking for lzma_auto_decoder in /usr/lib/x86_64-linux-gnu/liblzma.so\n",
      "-- Looking for lzma_auto_decoder in /usr/lib/x86_64-linux-gnu/liblzma.so - found\n",
      "-- Looking for lzma_easy_encoder in /usr/lib/x86_64-linux-gnu/liblzma.so\n",
      "-- Looking for lzma_easy_encoder in /usr/lib/x86_64-linux-gnu/liblzma.so - found\n",
      "-- Looking for lzma_lzma_preset in /usr/lib/x86_64-linux-gnu/liblzma.so\n",
      "-- Looking for lzma_lzma_preset in /usr/lib/x86_64-linux-gnu/liblzma.so - found\n",
      "-- Found LibLZMA: /usr/include (found version \"5.2.2\") \n",
      "-- Found Eigen3: /usr/include/eigen3 (Required is at least version \"2.91.0\") \n",
      "-- Found OpenMP_C: -fopenmp (found version \"4.5\") \n",
      "-- Found OpenMP_CXX: -fopenmp (found version \"4.5\") \n",
      "-- Found OpenMP: TRUE (found version \"4.5\")  \n",
      "-- Configuring done\n",
      "-- Generating done\n",
      "-- Build files have been written to: /gdrive/My Drive/Colab Notebooks/nlp_task/kenlm/kenlm/build\n",
      "\u001b[35m\u001b[1mScanning dependencies of target kenlm_util\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target kenlm_interpolate\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target kenlm_filter\u001b[0m\n",
      "[  0%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/kenlm_filter.dir/vocab.cc.o\u001b[0m\n",
      "[  0%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/kenlm_filter.dir/arpa_io.cc.o\u001b[0m\n",
      "[  1%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/kenlm_filter.dir/phrase.cc.o\u001b[0m\n",
      "[  2%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/bignum-dtoa.cc.o\u001b[0m\n",
      "[  5%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/cached-powers.cc.o\u001b[0m\n",
      "[  5%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/diy-fp.cc.o\u001b[0m\n",
      "[  5%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/bignum.cc.o\u001b[0m\n",
      "[  6%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/double-conversion.cc.o\u001b[0m\n",
      "[  7%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/fast-dtoa.cc.o\u001b[0m\n",
      "[  8%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/backoff_reunification.cc.o\u001b[0m\n",
      "[  9%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/fixed-dtoa.cc.o\u001b[0m\n",
      "[ 10%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/double-conversion/strtod.cc.o\u001b[0m\n",
      "[ 11%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/chain.cc.o\u001b[0m\n",
      "[ 12%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/count_records.cc.o\u001b[0m\n",
      "[ 13%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/io.cc.o\u001b[0m\n",
      "[ 14%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/line_input.cc.o\u001b[0m\n",
      "[ 15%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/multi_progress.cc.o\u001b[0m\n",
      "[ 16%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/stream/rewindable_stream.cc.o\u001b[0m\n",
      "[ 17%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/bounded_sequence_encoding.cc.o\u001b[0m\n",
      "[ 18%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/bit_packing.cc.o\u001b[0m\n",
      "[ 19%] \u001b[32m\u001b[1mLinking CXX static library ../../lib/libkenlm_filter.a\u001b[0m\n",
      "[ 20%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/ersatz_progress.cc.o\u001b[0m\n",
      "[ 21%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/merge_probabilities.cc.o\u001b[0m\n",
      "[ 21%] Built target kenlm_filter\n",
      "[ 22%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/merge_vocab.cc.o\u001b[0m\n",
      "[ 23%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/normalize.cc.o\u001b[0m\n",
      "[ 25%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/exception.cc.o\u001b[0m\n",
      "[ 26%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/file.cc.o\u001b[0m\n",
      "[ 27%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/file_piece.cc.o\u001b[0m\n",
      "[ 28%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/float_to_string.cc.o\u001b[0m\n",
      "[ 29%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/integer_to_string.cc.o\u001b[0m\n",
      "[ 30%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/pipeline.cc.o\u001b[0m\n",
      "[ 31%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/mmap.cc.o\u001b[0m\n",
      "[ 32%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/murmur_hash.cc.o\u001b[0m\n",
      "[ 33%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/parallel_read.cc.o\u001b[0m\n",
      "[ 34%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/pool.cc.o\u001b[0m\n",
      "[ 35%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/read_compressed.cc.o\u001b[0m\n",
      "[ 36%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/scoped.cc.o\u001b[0m\n",
      "[ 37%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/spaces.cc.o\u001b[0m\n",
      "[ 38%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/string_piece.cc.o\u001b[0m\n",
      "[ 39%] \u001b[32mBuilding CXX object util/CMakeFiles/kenlm_util.dir/usage.cc.o\u001b[0m\n",
      "[ 40%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/split_worker.cc.o\u001b[0m\n",
      "[ 41%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/tune_derivatives.cc.o\u001b[0m\n",
      "[ 42%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/tune_instances.cc.o\u001b[0m\n",
      "[ 44%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/tune_weights.cc.o\u001b[0m\n",
      "[ 44%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/kenlm_interpolate.dir/universal_vocab.cc.o\u001b[0m\n",
      "[ 45%] \u001b[32m\u001b[1mLinking CXX static library ../lib/libkenlm_util.a\u001b[0m\n",
      "[ 45%] Built target kenlm_util\n",
      "\u001b[35m\u001b[1mScanning dependencies of target probing_hash_table_benchmark\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target kenlm\u001b[0m\n",
      "[ 46%] \u001b[32mBuilding CXX object util/CMakeFiles/probing_hash_table_benchmark.dir/probing_hash_table_benchmark_main.cc.o\u001b[0m\n",
      "[ 47%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/binary_format.cc.o\u001b[0m\n",
      "[ 48%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/bhiksha.cc.o\u001b[0m\n",
      "[ 50%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/config.cc.o\u001b[0m\n",
      "[ 51%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/lm_exception.cc.o\u001b[0m\n",
      "[ 52%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/model.cc.o\u001b[0m\n",
      "[ 53%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/quantize.cc.o\u001b[0m\n",
      "[ 54%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/read_arpa.cc.o\u001b[0m\n",
      "[ 55%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/search_hashed.cc.o\u001b[0m\n",
      "[ 56%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/search_trie.cc.o\u001b[0m\n",
      "[ 57%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/sizes.cc.o\u001b[0m\n",
      "[ 58%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/trie.cc.o\u001b[0m\n",
      "[ 59%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/trie_sort.cc.o\u001b[0m\n",
      "[ 60%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/value_build.cc.o\u001b[0m\n",
      "[ 61%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/virtual_interface.cc.o\u001b[0m\n",
      "[ 62%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/vocab.cc.o\u001b[0m\n",
      "[ 63%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/model_buffer.cc.o\u001b[0m\n",
      "[ 64%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/print.cc.o\u001b[0m\n",
      "[ 65%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/renumber.cc.o\u001b[0m\n",
      "[ 66%] \u001b[32m\u001b[1mLinking CXX static library ../../lib/libkenlm_interpolate.a\u001b[0m\n",
      "[ 67%] \u001b[32m\u001b[1mLinking CXX executable ../bin/probing_hash_table_benchmark\u001b[0m\n",
      "[ 67%] Built target probing_hash_table_benchmark\n",
      "[ 68%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm.dir/common/size_option.cc.o\u001b[0m\n",
      "[ 68%] Built target kenlm_interpolate\n",
      "[ 69%] \u001b[32m\u001b[1mLinking CXX static library ../lib/libkenlm.a\u001b[0m\n",
      "[ 69%] Built target kenlm\n",
      "\u001b[35m\u001b[1mScanning dependencies of target build_binary\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target fragment\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target query\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target streaming_example\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target kenlm_benchmark\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target filter\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target kenlm_builder\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target phrase_table_vocab\u001b[0m\n",
      "[ 70%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/phrase_table_vocab.dir/phrase_table_vocab_main.cc.o\u001b[0m\n",
      "[ 71%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/streaming_example.dir/streaming_example_main.cc.o\u001b[0m\n",
      "[ 72%] \u001b[32mBuilding CXX object lm/filter/CMakeFiles/filter.dir/filter_main.cc.o\u001b[0m\n",
      "[ 75%] \u001b[32mBuilding CXX object lm/CMakeFiles/fragment.dir/fragment_main.cc.o\u001b[0m\n",
      "[ 75%] \u001b[32mBuilding CXX object lm/CMakeFiles/build_binary.dir/build_binary_main.cc.o\u001b[0m\n",
      "[ 76%] \u001b[32mBuilding CXX object lm/CMakeFiles/kenlm_benchmark.dir/kenlm_benchmark_main.cc.o\u001b[0m\n",
      "[ 77%] \u001b[32mBuilding CXX object lm/CMakeFiles/query.dir/query_main.cc.o\u001b[0m\n",
      "[ 78%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/adjust_counts.cc.o\u001b[0m\n",
      "[ 79%] \u001b[32m\u001b[1mLinking CXX executable ../bin/fragment\u001b[0m\n",
      "[ 80%] \u001b[32m\u001b[1mLinking CXX executable ../bin/build_binary\u001b[0m\n",
      "[ 80%] Built target fragment\n",
      "\u001b[35m\u001b[1mScanning dependencies of target interpolate\u001b[0m\n",
      "[ 80%] Built target build_binary\n",
      "[ 81%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/corpus_count.cc.o\u001b[0m\n",
      "[ 82%] \u001b[32mBuilding CXX object lm/interpolate/CMakeFiles/interpolate.dir/interpolate_main.cc.o\u001b[0m\n",
      "[ 83%] \u001b[32m\u001b[1mLinking CXX executable ../bin/query\u001b[0m\n",
      "[ 83%] Built target query\n",
      "[ 84%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/initial_probabilities.cc.o\u001b[0m\n",
      "[ 85%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/phrase_table_vocab\u001b[0m\n",
      "[ 85%] Built target phrase_table_vocab\n",
      "[ 86%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/interpolate.cc.o\u001b[0m\n",
      "[ 87%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/output.cc.o\u001b[0m\n",
      "[ 88%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/kenlm_builder.dir/pipeline.cc.o\u001b[0m\n",
      "[ 89%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/streaming_example\u001b[0m\n",
      "[ 89%] Built target streaming_example\n",
      "[ 90%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/interpolate\u001b[0m\n",
      "[ 90%] Built target interpolate\n",
      "[ 91%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/filter\u001b[0m\n",
      "[ 91%] Built target filter\n",
      "[ 92%] \u001b[32m\u001b[1mLinking CXX executable ../bin/kenlm_benchmark\u001b[0m\n",
      "[ 92%] Built target kenlm_benchmark\n",
      "[ 93%] \u001b[32m\u001b[1mLinking CXX static library ../../lib/libkenlm_builder.a\u001b[0m\n",
      "[ 93%] Built target kenlm_builder\n",
      "\u001b[35m\u001b[1mScanning dependencies of target count_ngrams\u001b[0m\n",
      "\u001b[35m\u001b[1mScanning dependencies of target lmplz\u001b[0m\n",
      "[ 95%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/lmplz.dir/lmplz_main.cc.o\u001b[0m\n",
      "[ 95%] \u001b[32mBuilding CXX object lm/builder/CMakeFiles/count_ngrams.dir/count_ngrams_main.cc.o\u001b[0m\n",
      "[ 96%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/lmplz\u001b[0m\n",
      "[ 96%] Built target lmplz\n",
      "[ 97%] \u001b[32m\u001b[1mLinking CXX executable ../../bin/count_ngrams\u001b[0m\n",
      "[ 97%] Built target count_ngrams\n"
     ]
    }
   ],
   "source": [
    "# 安装依赖\n",
    "!apt install libboost-all-dev\n",
    "!apt install libbz2-dev\n",
    "!apt install libeigen3-dev\n",
    "\n",
    "# 下载KenLM编译\n",
    "!wget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz\n",
    "!mkdir kenlm/build\n",
    "!cd kenlm/build && cmake .. && make -j8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4528,
     "status": "ok",
     "timestamp": 1592668514803,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "41whsj7Sber9",
    "outputId": "4ab215f7-2947-4e15-c07c-4e93480f0bd8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 我 们 去 抢 劫 吧\r\n",
      " 我 们 去 抢 劫 吧 哎 ， 我 最 腻 味 这 种 没 有 技 术 含 量 的 活 儿 ～\r\n",
      " 王 。\r\n",
      " 王 。 哎 ， 叫 朕 何 事 ？\r\n",
      " 喜 欢 什 么 电 影\r\n"
     ]
    }
   ],
   "source": [
    "!head -5 ./corpus_task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7972,
     "status": "ok",
     "timestamp": 1592668798518,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "UgpWDAGcdRAM",
    "outputId": "c458c749-c582-4d04-f092-00ebef3179f3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/gdrive/My Drive/Colab Notebooks/nlp_task/kenlm\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 647
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8170,
     "status": "ok",
     "timestamp": 1592668831123,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "DPEJsZbRcGjL",
    "outputId": "86f7de75-e210-4a2e-d88e-48e85618eefe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== 1/5 Counting and sorting n-grams ===\n",
      "Reading /gdrive/My Drive/Colab Notebooks/nlp_task/kenlm/corpus_task\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "tcmalloc: large alloc 2514853888 bytes == 0x55a4e4348000 @  0x7fad943651e7 0x55a4e18c3772 0x55a4e1857358 0x55a4e1836290 0x55a4e1822096 0x7fad924feb97 0x55a4e1823ada\n",
      "tcmalloc: large alloc 8382840832 bytes == 0x55a57a1a2000 @  0x7fad943651e7 0x55a4e18c3772 0x55a4e18ad7aa 0x55a4e18ae1c8 0x55a4e18362ad 0x55a4e1822096 0x7fad924feb97 0x55a4e1823ada\n",
      "****************************************************************************************************\n",
      "Unigram tokens 3073460 types 4871\n",
      "=== 2/5 Calculating and sorting adjusted counts ===\n",
      "Chain sizes: 1:58452 2:3799231744 3:7123559424\n",
      "tcmalloc: large alloc 7123566592 bytes == 0x55a4e4348000 @  0x7fad943651e7 0x55a4e18c3772 0x55a4e18ad7aa 0x55a4e18ae1c8 0x55a4e183684e 0x55a4e1822096 0x7fad924feb97 0x55a4e1823ada\n",
      "tcmalloc: large alloc 3799236608 bytes == 0x55a76e4ce000 @  0x7fad943651e7 0x55a4e18c3772 0x55a4e18ad7aa 0x55a4e18ae1c8 0x55a4e1836c3d 0x55a4e1822096 0x7fad924feb97 0x55a4e1823ada\n",
      "Statistics:\n",
      "1 4871 D1=0.609884 D2=0.960505 D3+=1.40112\n",
      "2 146052 D1=0.687171 D2=1.16593 D3+=1.41673\n",
      "3 419340 D1=0.414095 D2=1.44188 D3+=1.79083\n",
      "Memory estimate for binary LM:\n",
      "type       kB\n",
      "probing 10917 assuming -p 1.5\n",
      "probing 11792 assuming -r models -p 1.5\n",
      "trie     4060 without quantization\n",
      "trie     2047 assuming -q 8 -b 8 quantization \n",
      "trie     3871 assuming -a 22 array pointer compression\n",
      "trie     1859 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
      "=== 3/5 Calculating and sorting initial probabilities ===\n",
      "Chain sizes: 1:58452 2:2336832 3:8386800\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "####################################################################################################\n",
      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
      "Chain sizes: 1:58452 2:2336832 3:8386800\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "####################################################################################################\n",
      "=== 5/5 Writing ARPA model ===\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "****************************************************************************************************\n",
      "Name:lmplz\tVmPeak:14638764 kB\tVmRSS:2498916 kB\tRSSMax:2507008 kB\tuser:1.00564\tsys:1.2447\tCPU:2.25038\treal:3.60978\n"
     ]
    }
   ],
   "source": [
    "# 3-gram语言模型\n",
    "!./kenlm/build/bin/lmplz -o 3 <corpus_task> lm_3.arpa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0_jrjXF-eGB_"
   },
   "source": [
    "对训练得到的文件进行压缩：将arpa文件转换为binary文件，这样可以对arpa文件进行压缩和序列化，提高后续在python中加载的速度。针对我们训练的到的 lm.arpa 文件其转换命令为："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7795,
     "status": "ok",
     "timestamp": 1592668982384,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "Ju-qMk4sdEgW",
    "outputId": "7b4e64da-4112-404f-9a4d-5ba1e4d59e1d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lm_3.arpa\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "****************************************************************************************************\n",
      "SUCCESS\n"
     ]
    }
   ],
   "source": [
    "!./kenlm/build/bin/build_binary -s lm_3.arpa lm.bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 43654,
     "status": "ok",
     "timestamp": 1592669069978,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "9eDS0IScd-jE",
    "outputId": "f4b80b48-4ac3-4f62-9f0f-398c02eca800"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting https://github.com/kpu/kenlm/archive/master.zip\n",
      "\u001b[?25l  Downloading https://github.com/kpu/kenlm/archive/master.zip\n",
      "\r",
      "\u001b[K     - 10kB 122kB/s\r",
      "\u001b[K     \\ 20kB 245kB/s\r",
      "\u001b[K     | 30kB 173kB/s\r",
      "\u001b[K     / 40kB 230kB/s\r",
      "\u001b[K     - 51kB 288kB/s\r",
      "\u001b[K     \\ 61kB 345kB/s\r",
      "\u001b[K     | 71kB 274kB/s\r",
      "\u001b[K     / 81kB 313kB/s\r",
      "\u001b[K     - 92kB 352kB/s\r",
      "\u001b[K     \\ 102kB 391kB/s\r",
      "\u001b[K     | 112kB 391kB/s\r",
      "\u001b[K     / 122kB 391kB/s\r",
      "\u001b[K     - 133kB 391kB/s\r",
      "\u001b[K     \\ 143kB 391kB/s\r",
      "\u001b[K     | 153kB 391kB/s\r",
      "\u001b[K     / 163kB 391kB/s\r",
      "\u001b[K     - 174kB 391kB/s\r",
      "\u001b[K     \\ 184kB 391kB/s\r",
      "\u001b[K     | 194kB 391kB/s\r",
      "\u001b[K     / 204kB 391kB/s\r",
      "\u001b[K     - 215kB 391kB/s\r",
      "\u001b[K     \\ 225kB 391kB/s\r",
      "\u001b[K     | 235kB 391kB/s\r",
      "\u001b[K     / 245kB 391kB/s\r",
      "\u001b[K     - 256kB 391kB/s\r",
      "\u001b[K     \\ 266kB 391kB/s\r",
      "\u001b[K     | 276kB 391kB/s\r",
      "\u001b[K     / 286kB 391kB/s\r",
      "\u001b[K     - 296kB 391kB/s\r",
      "\u001b[K     \\ 307kB 391kB/s\r",
      "\u001b[K     | 317kB 391kB/s\r",
      "\u001b[K     / 327kB 391kB/s\r",
      "\u001b[K     - 337kB 391kB/s\r",
      "\u001b[K     \\ 348kB 391kB/s\r",
      "\u001b[K     | 358kB 391kB/s\r",
      "\u001b[K     / 368kB 391kB/s\r",
      "\u001b[K     - 378kB 391kB/s\r",
      "\u001b[K     \\ 389kB 391kB/s\r",
      "\u001b[K     | 399kB 391kB/s\r",
      "\u001b[K     / 409kB 391kB/s\r",
      "\u001b[K     - 419kB 391kB/s\r",
      "\u001b[K     \\ 430kB 391kB/s\r",
      "\u001b[K     | 440kB 391kB/s\r",
      "\u001b[K     / 450kB 391kB/s\r",
      "\u001b[K     - 460kB 391kB/s\r",
      "\u001b[K     \\ 471kB 391kB/s\r",
      "\u001b[K     | 481kB 391kB/s\r",
      "\u001b[K     / 491kB 391kB/s\r",
      "\u001b[K     - 501kB 391kB/s\r",
      "\u001b[K     \\ 512kB 391kB/s\r",
      "\u001b[K     | 522kB 391kB/s\r",
      "\u001b[K     / 532kB 391kB/s\r",
      "\u001b[K     - 542kB 391kB/s\r",
      "\u001b[K     \\ 552kB 391kB/s\r",
      "\u001b[K     | 563kB 391kB/s\r",
      "\u001b[K     / 573kB 391kB/s\r",
      "\u001b[K     - 583kB 391kB/s\r",
      "\u001b[K     \\ 593kB 391kB/s\r",
      "\u001b[K     | 604kB 391kB/s\r",
      "\u001b[K     / 614kB 391kB/s\r",
      "\u001b[K     - 624kB 391kB/s\r",
      "\u001b[K     \\ 634kB 391kB/s\r",
      "\u001b[K     | 645kB 391kB/s\r",
      "\u001b[K     / 655kB 391kB/s\r",
      "\u001b[K     - 665kB 391kB/s\r",
      "\u001b[K     \\ 675kB 391kB/s\r",
      "\u001b[K     | 686kB 391kB/s\r",
      "\u001b[K     / 696kB 391kB/s\r",
      "\u001b[K     - 706kB 391kB/s\r",
      "\u001b[K     \\ 716kB 391kB/s\r",
      "\u001b[K     | 727kB 391kB/s\r",
      "\u001b[K     / 737kB 391kB/s\r",
      "\u001b[K     - 747kB 391kB/s\r",
      "\u001b[K     \\ 757kB 391kB/s\r",
      "\u001b[K     | 768kB 391kB/s\r",
      "\u001b[K     / 778kB 391kB/s\r",
      "\u001b[K     - 788kB 391kB/s\r",
      "\u001b[K     \\ 798kB 391kB/s\r",
      "\u001b[K     | 808kB 391kB/s\r",
      "\u001b[K     / 819kB 391kB/s\r",
      "\u001b[K     - 829kB 391kB/s\r",
      "\u001b[K     \\ 839kB 391kB/s\r",
      "\u001b[K     | 849kB 391kB/s\r",
      "\u001b[K     / 860kB 391kB/s\r",
      "\u001b[K     - 870kB 391kB/s\r",
      "\u001b[K     \\ 880kB 391kB/s\r",
      "\u001b[K     | 890kB 391kB/s\r",
      "\u001b[K     / 901kB 391kB/s\r",
      "\u001b[K     - 911kB 391kB/s\r",
      "\u001b[K     \\ 921kB 391kB/s\r",
      "\u001b[K     | 931kB 391kB/s\r",
      "\u001b[K     / 942kB 391kB/s\r",
      "\u001b[K     - 952kB 391kB/s\r",
      "\u001b[K     \\ 962kB 391kB/s\r",
      "\u001b[K     | 972kB 391kB/s\r",
      "\u001b[K     / 983kB 391kB/s\r",
      "\u001b[K     - 993kB 391kB/s\r",
      "\u001b[K     \\ 1.0MB 391kB/s\r",
      "\u001b[K     | 1.0MB 391kB/s\r",
      "\u001b[K     / 1.0MB 391kB/s\r",
      "\u001b[K     - 1.0MB 391kB/s\r",
      "\u001b[K     \\ 1.0MB 391kB/s\r",
      "\u001b[K     | 1.1MB 391kB/s\r",
      "\u001b[K     / 1.1MB 391kB/s\r",
      "\u001b[K     - 1.1MB 391kB/s\r",
      "\u001b[K     \\ 1.1MB 391kB/s\r",
      "\u001b[K     | 1.1MB 391kB/s\r",
      "\u001b[K     / 1.1MB 391kB/s\r",
      "\u001b[K     - 1.1MB 391kB/s\r",
      "\u001b[K     \\ 1.1MB 391kB/s\r",
      "\u001b[K     | 1.1MB 391kB/s\r",
      "\u001b[K     / 1.1MB 391kB/s\r",
      "\u001b[K     - 1.2MB 391kB/s\r",
      "\u001b[K     \\ 1.2MB 391kB/s\r",
      "\u001b[K     | 1.2MB 391kB/s\r",
      "\u001b[K     / 1.2MB 391kB/s\r",
      "\u001b[K     - 1.2MB 391kB/s\r",
      "\u001b[K     \\ 1.2MB 391kB/s\r",
      "\u001b[K     | 1.2MB 391kB/s\r",
      "\u001b[K     / 1.2MB 391kB/s\r",
      "\u001b[K     - 1.2MB 391kB/s\r",
      "\u001b[K     \\ 1.2MB 391kB/s\r",
      "\u001b[K     | 1.3MB 391kB/s\r",
      "\u001b[K     / 1.3MB 391kB/s\r",
      "\u001b[K     - 1.3MB 391kB/s\r",
      "\u001b[K     \\ 1.3MB 391kB/s\r",
      "\u001b[K     | 1.3MB 391kB/s\r",
      "\u001b[K     / 1.3MB 391kB/s\r",
      "\u001b[K     - 1.3MB 391kB/s\r",
      "\u001b[K     \\ 1.3MB 391kB/s\r",
      "\u001b[K     | 1.3MB 391kB/s\r",
      "\u001b[K     / 1.4MB 391kB/s\r",
      "\u001b[K     - 1.4MB 391kB/s\r",
      "\u001b[K     \\ 1.4MB 391kB/s\r",
      "\u001b[K     | 1.4MB 391kB/s\r",
      "\u001b[K     / 1.4MB 391kB/s\r",
      "\u001b[K     - 1.4MB 391kB/s\r",
      "\u001b[K     \\ 1.4MB 391kB/s\r",
      "\u001b[K     | 1.4MB 391kB/s\r",
      "\u001b[K     / 1.4MB 391kB/s\r",
      "\u001b[K     - 1.4MB 391kB/s\r",
      "\u001b[K     \\ 1.5MB 391kB/s\r",
      "\u001b[K     | 1.5MB 391kB/s\r",
      "\u001b[K     / 1.5MB 391kB/s\r",
      "\u001b[K     - 1.5MB 391kB/s\r",
      "\u001b[K     \\ 1.5MB 391kB/s\r",
      "\u001b[K     | 1.5MB 391kB/s\r",
      "\u001b[K     / 1.5MB 391kB/s\r",
      "\u001b[K     - 1.5MB 391kB/s\r",
      "\u001b[K     \\ 1.5MB 391kB/s\r",
      "\u001b[K     | 1.5MB 391kB/s\r",
      "\u001b[K     / 1.6MB 391kB/s\r",
      "\u001b[K     - 1.6MB 391kB/s\r",
      "\u001b[K     \\ 1.6MB 391kB/s\r",
      "\u001b[K     | 1.6MB 391kB/s\r",
      "\u001b[K     / 1.6MB 391kB/s\r",
      "\u001b[K     - 1.6MB 391kB/s\r",
      "\u001b[K     \\ 1.6MB 391kB/s\r",
      "\u001b[K     | 1.6MB 391kB/s\r",
      "\u001b[K     / 1.6MB 391kB/s\r",
      "\u001b[K     - 1.6MB 391kB/s\r",
      "\u001b[K     \\ 1.7MB 391kB/s\r",
      "\u001b[K     | 1.7MB 391kB/s\r",
      "\u001b[K     / 1.7MB 391kB/s\r",
      "\u001b[K     - 1.7MB 391kB/s\r",
      "\u001b[K     \\ 1.7MB 391kB/s\r",
      "\u001b[K     | 1.7MB 391kB/s\r",
      "\u001b[K     / 1.7MB 391kB/s\r",
      "\u001b[K     - 1.7MB 391kB/s\r",
      "\u001b[K     \\ 1.7MB 391kB/s\r",
      "\u001b[K     | 1.8MB 391kB/s\r",
      "\u001b[K     / 1.8MB 391kB/s\r",
      "\u001b[K     - 1.8MB 391kB/s\r",
      "\u001b[K     \\ 1.8MB 391kB/s\r",
      "\u001b[K     | 1.8MB 391kB/s\r",
      "\u001b[K     / 1.8MB 391kB/s\r",
      "\u001b[K     - 1.8MB 391kB/s\r",
      "\u001b[K     \\ 1.8MB 391kB/s\r",
      "\u001b[K     | 1.8MB 391kB/s\r",
      "\u001b[K     / 1.8MB 391kB/s\r",
      "\u001b[K     - 1.9MB 391kB/s\r",
      "\u001b[K     \\ 1.9MB 391kB/s\r",
      "\u001b[K     | 1.9MB 391kB/s\r",
      "\u001b[K     / 1.9MB 391kB/s\r",
      "\u001b[K     - 1.9MB 391kB/s\r",
      "\u001b[K     \\ 1.9MB 391kB/s\r",
      "\u001b[K     | 1.9MB 391kB/s\r",
      "\u001b[K     / 1.9MB 391kB/s\r",
      "\u001b[K     - 1.9MB 391kB/s\r",
      "\u001b[K     \\ 1.9MB 391kB/s\r",
      "\u001b[K     | 2.0MB 391kB/s\r",
      "\u001b[K     / 2.0MB 391kB/s\r",
      "\u001b[K     - 2.0MB 391kB/s\r",
      "\u001b[K     \\ 2.0MB 391kB/s\r",
      "\u001b[K     | 2.0MB 391kB/s\r",
      "\u001b[K     / 2.0MB 391kB/s\r",
      "\u001b[K     - 2.0MB 391kB/s\r",
      "\u001b[K     \\ 2.0MB 391kB/s\r",
      "\u001b[K     | 2.0MB 391kB/s\r",
      "\u001b[K     / 2.0MB 391kB/s\r",
      "\u001b[K     - 2.1MB 391kB/s\r",
      "\u001b[K     \\ 2.1MB 391kB/s\r",
      "\u001b[K     | 2.1MB 391kB/s\r",
      "\u001b[K     / 2.1MB 391kB/s\r",
      "\u001b[K     - 2.1MB 391kB/s\r",
      "\u001b[K     \\ 2.1MB 391kB/s\r",
      "\u001b[K     | 2.1MB 391kB/s\r",
      "\u001b[K     / 2.1MB 391kB/s\r",
      "\u001b[K     - 2.1MB 391kB/s\r",
      "\u001b[K     \\ 2.2MB 391kB/s\r",
      "\u001b[K     | 2.2MB 391kB/s\r",
      "\u001b[K     / 2.2MB 391kB/s\r",
      "\u001b[K     - 2.2MB 391kB/s\r",
      "\u001b[K     \\ 2.2MB 391kB/s\r",
      "\u001b[K     | 2.2MB 391kB/s\r",
      "\u001b[K     / 2.2MB 391kB/s\r",
      "\u001b[K     - 2.2MB 391kB/s\r",
      "\u001b[K     \\ 2.2MB 391kB/s\r",
      "\u001b[K     | 2.2MB 391kB/s\r",
      "\u001b[K     / 2.3MB 391kB/s\r",
      "\u001b[K     - 2.3MB 391kB/s\r",
      "\u001b[K     \\ 2.3MB 391kB/s\r",
      "\u001b[K     | 2.3MB 391kB/s\r",
      "\u001b[K     / 2.3MB 391kB/s\r",
      "\u001b[K     - 2.3MB 391kB/s\r",
      "\u001b[K     \\ 2.3MB 391kB/s\r",
      "\u001b[K     | 2.3MB 391kB/s\r",
      "\u001b[K     / 2.3MB 391kB/s\r",
      "\u001b[K     - 2.3MB 391kB/s\r",
      "\u001b[K     \\ 2.4MB 391kB/s\r",
      "\u001b[K     | 2.4MB 391kB/s\r",
      "\u001b[K     / 2.4MB 391kB/s\r",
      "\u001b[K     - 2.4MB 391kB/s\r",
      "\u001b[K     \\ 2.4MB 391kB/s\r",
      "\u001b[K     | 2.4MB 391kB/s\r",
      "\u001b[K     / 2.4MB 391kB/s\r",
      "\u001b[K     - 2.4MB 391kB/s\r",
      "\u001b[K     \\ 2.4MB 391kB/s\r",
      "\u001b[K     | 2.4MB 391kB/s\r",
      "\u001b[K     / 2.5MB 391kB/s\r",
      "\u001b[K     - 2.5MB 391kB/s\r",
      "\u001b[K     \\ 2.5MB 391kB/s\r",
      "\u001b[K     | 2.5MB 391kB/s\r",
      "\u001b[K     / 2.5MB 391kB/s\r",
      "\u001b[K     - 2.5MB 391kB/s\r",
      "\u001b[K     \\ 2.5MB 391kB/s\r",
      "\u001b[K     | 2.5MB 391kB/s\r",
      "\u001b[K     / 2.5MB 391kB/s\r",
      "\u001b[K     - 2.5MB 391kB/s\r",
      "\u001b[K     \\ 2.6MB 391kB/s\r",
      "\u001b[K     | 2.6MB 391kB/s\r",
      "\u001b[K     / 2.6MB 391kB/s\r",
      "\u001b[K     - 2.6MB 391kB/s\r",
      "\u001b[K     \\ 2.6MB 391kB/s\r",
      "\u001b[K     | 2.6MB 391kB/s\r",
      "\u001b[K     / 2.6MB 391kB/s\r",
      "\u001b[K     - 2.6MB 391kB/s\r",
      "\u001b[K     \\ 2.6MB 391kB/s\r",
      "\u001b[K     | 2.7MB 391kB/s\r",
      "\u001b[K     / 2.7MB 391kB/s\r",
      "\u001b[K     - 2.7MB 391kB/s\r",
      "\u001b[K     \\ 2.7MB 391kB/s\r",
      "\u001b[K     | 2.7MB 391kB/s\r",
      "\u001b[K     / 2.7MB 391kB/s\r",
      "\u001b[K     - 2.7MB 391kB/s\r",
      "\u001b[K     \\ 2.7MB 391kB/s\r",
      "\u001b[K     | 2.7MB 391kB/s\r",
      "\u001b[K     / 2.7MB 391kB/s\r",
      "\u001b[K     - 2.8MB 391kB/s\r",
      "\u001b[K     \\ 2.8MB 391kB/s\r",
      "\u001b[K     | 2.8MB 391kB/s\r",
      "\u001b[K     / 2.8MB 391kB/s\r",
      "\u001b[K     - 2.8MB 391kB/s\r",
      "\u001b[K     \\ 2.8MB 391kB/s\n",
      "\u001b[?25hBuilding wheels for collected packages: kenlm\n",
      "  Building wheel for kenlm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for kenlm: filename=kenlm-0.0.0-cp36-cp36m-linux_x86_64.whl size=2330547 sha256=e2d49e0206abd28c20bc8d413d5445226259003d33383b518fd48c00578666b1\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-m9lk4anj/wheels/2d/32/73/e3093c9d11dc8abf79c156a4db1a1c5631428059d4f9ff2cba\n",
      "Successfully built kenlm\n",
      "Installing collected packages: kenlm\n",
      "Successfully installed kenlm-0.0.0\n"
     ]
    }
   ],
   "source": [
    "!pip install https://github.com/kpu/kenlm/archive/master.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1102,
     "status": "ok",
     "timestamp": 1592669073795,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "sBb8EFPaeLev"
   },
   "outputs": [],
   "source": [
    "# 加载模型\n",
    "import kenlm\n",
    "\n",
    "# 将文件导入到 kenlm 语言模型中\n",
    "model = kenlm.LanguageModel(\"./lm.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1142,
     "status": "ok",
     "timestamp": 1592669121096,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "CD6tBNjJeWzi",
    "outputId": "9f8b6fca-8612-4bb1-ca22-ac229f8f38f0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-7.646023750305176"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"怎 么 办 理 信 用 卡\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1194,
     "status": "ok",
     "timestamp": 1592669151548,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "Gm7t9wJTeiV4",
    "outputId": "7998cf40-04d2-4588-e3a2-dc84c5b65788"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-14.39771556854248"
      ]
     },
     "execution_count": 16,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"怎 么 办 理 新 用 卡\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1169,
     "status": "ok",
     "timestamp": 1592669170384,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "Pc2PO1LcepxH",
    "outputId": "da65237a-a235-465e-ccde-5d791ae553ba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-8.711942672729492"
      ]
     },
     "execution_count": 17,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"怎 么 办 理 储 蓄 卡\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1242,
     "status": "ok",
     "timestamp": 1592669209397,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "_jI9Nva9euXv",
    "outputId": "705b16f1-68cf-4e55-9ef7-c216d2216d80"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-8.195541381835938"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"龙 卡 是 什 么\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1135,
     "status": "ok",
     "timestamp": 1592669238566,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "JY0ILAR2e34Q",
    "outputId": "1fc37ccf-026e-4fc0-f83e-5a1df97146c8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-12.113533020019531"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"农 卡 是 什 么\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1141,
     "status": "ok",
     "timestamp": 1592669255604,
     "user": {
      "displayName": "Racle He",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhNRi7DUXsEvpMRqVqHTUF_Oen4KW7kU7MKQekk=s64",
      "userId": "10673173760458122172"
     },
     "user_tz": -480
    },
    "id": "6uzusCM8e_Bq",
    "outputId": "764d4747-8d57-4bf3-9349-2e5c527420f1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-12.968329429626465"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"卡 农 是 什 么\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8ns-V6h9fF5m"
   },
   "source": [
    "Nice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拼音转换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:19.308139Z",
     "start_time": "2020-06-22T01:57:18.561586Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "nDqOLwo4fDL3"
   },
   "outputs": [],
   "source": [
    "from pypinyin import pinyin, lazy_pinyin, Style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:35:12.339208Z",
     "start_time": "2020-06-21T07:35:12.325176Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['zhōng'], ['xīn']]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinyin('中心')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:33:48.912096Z",
     "start_time": "2020-06-21T07:33:48.894096Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['zhōng', 'zhòng'], ['xīn']]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinyin('中心', heteronym=True)  # 启用多音字模式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:33:54.265316Z",
     "start_time": "2020-06-21T07:33:54.255317Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['z'], ['x']]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinyin('中心', style=Style.FIRST_LETTER)  # 设置拼音风格"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:34:57.868046Z",
     "start_time": "2020-06-21T07:34:57.860045Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['zho1ng', 'zho4ng'], ['xi1n']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinyin('中心', style=Style.TONE2, heteronym=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:35:22.593455Z",
     "start_time": "2020-06-21T07:35:22.573421Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['zhong1', 'zhong4'], ['xin1']]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinyin('中心', style=Style.TONE3, heteronym=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:35:29.338107Z",
     "start_time": "2020-06-21T07:35:29.333107Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['ㄓㄨㄥ'], ['ㄒㄧㄣ']]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinyin('中心', style=Style.BOPOMOFO)  # 注音风格"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T07:36:00.086064Z",
     "start_time": "2020-06-21T07:36:00.071038Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['yin', 'hang']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lazy_pinyin('银行')  # 不考虑多音字的情况"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trie测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:29.080039Z",
     "start_time": "2020-06-22T01:57:29.076047Z"
    }
   },
   "outputs": [],
   "source": [
    "# import os\n",
    "# import stat\n",
    "# import sys\n",
    "\n",
    "# import pygtrie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/ethereum/py-trie bug一堆"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:33.363406Z",
     "start_time": "2020-06-22T01:57:33.298407Z"
    }
   },
   "outputs": [],
   "source": [
    "import string\n",
    "import datrie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:48.851666Z",
     "start_time": "2020-06-22T01:57:48.795703Z"
    }
   },
   "outputs": [],
   "source": [
    "trie = datrie.Trie(string.ascii_lowercase)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:54.217965Z",
     "start_time": "2020-06-22T01:57:54.209958Z"
    }
   },
   "outputs": [],
   "source": [
    "trie[u'foo'] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:54.644874Z",
     "start_time": "2020-06-22T01:57:54.633878Z"
    }
   },
   "outputs": [],
   "source": [
    "trie[u'foo'] = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:55.165875Z",
     "start_time": "2020-06-22T01:57:55.146875Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trie[u'foo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:55.415875Z",
     "start_time": "2020-06-22T01:57:55.412876Z"
    }
   },
   "outputs": [],
   "source": [
    "s = u'你好'.encode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:56.069387Z",
     "start_time": "2020-06-22T01:57:56.058080Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'\\xe4\\xbd\\xa0\\xe5\\xa5\\xbd'\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:56.701069Z",
     "start_time": "2020-06-22T01:57:56.682069Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"\\\\u4f60\\\\u597d\"'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "json.dumps('你好')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:57:57.586682Z",
     "start_time": "2020-06-22T01:57:57.542684Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\\\u4f60\\\\u597d'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'你好'.encode(\"unicode_escape\").decode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理语料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T03:41:41.060288Z",
     "start_time": "2020-06-22T03:41:41.032287Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils import save_pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:58:00.972905Z",
     "start_time": "2020-06-22T01:58:00.968903Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def token(string):\n",
    "        return re.findall(r'[\\d|\\w]+', string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T03:40:30.712819Z",
     "start_time": "2020-06-22T03:40:28.809565Z"
    }
   },
   "outputs": [],
   "source": [
    "words = {}\n",
    "words_frequnece = {}\n",
    "# 文本文件经过预处理: language_model_data_process\n",
    "with open('data/corpus_task', 'r', encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        line = token(line)\n",
    "#         print(line)\n",
    "#         break\n",
    "        for w in line:\n",
    "            if w in words: \n",
    "                words_frequnece[w] = words_frequnece[w] + 1\n",
    "                continue\n",
    "            words_frequnece[w] = 1\n",
    "            pin = lazy_pinyin(w)[0]\n",
    "            words[w] = pin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T03:42:21.766581Z",
     "start_time": "2020-06-22T03:42:21.757587Z"
    }
   },
   "outputs": [],
   "source": [
    "save_pickle('data/words_frequnece.bin', words_frequnece)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T03:38:36.076774Z",
     "start_time": "2020-06-22T03:38:36.058800Z"
    }
   },
   "outputs": [],
   "source": [
    "# def ch_char(string):\n",
    "#     return re.findall(r'[\\u4e00-\\u9fa5]', string)\n",
    "\n",
    "# 增加一点汉字集, 非必要\n",
    "# with open('data/raw_article_9k.txt', 'r', encoding='utf-8') as f:\n",
    "# #     print(f.readline())\n",
    "# #     print(ch_char(f.readline()))\n",
    "#     for line in f.readlines():\n",
    "#         line = ch_char(line)\n",
    "#         for w in line:\n",
    "#             if w in words: continue\n",
    "#             pin = lazy_pinyin(w)[0]\n",
    "#             words[w] = pin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:58:11.937803Z",
     "start_time": "2020-06-22T01:58:11.928534Z"
    }
   },
   "outputs": [],
   "source": [
    "import string\n",
    "import datrie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:58:12.711635Z",
     "start_time": "2020-06-22T01:58:12.706637Z"
    }
   },
   "outputs": [],
   "source": [
    "trie = datrie.Trie(string.ascii_lowercase + string.digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:58:15.980054Z",
     "start_time": "2020-06-22T01:58:15.921542Z"
    }
   },
   "outputs": [],
   "source": [
    "for w, pin in words.items():\n",
    "    if pin in trie and w == trie[pin]:\n",
    "        continue\n",
    "    # 相同拼音处理\n",
    "    if pin in trie and w != trie[pin]:\n",
    "        i = 1\n",
    "        flag = True\n",
    "        while flag:\n",
    "            key = pin + str(i)\n",
    "            if key not in trie:\n",
    "                flag = False\n",
    "            i += 1\n",
    "        trie[key] = w\n",
    "    else:\n",
    "        trie[pin] = w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:58:18.998761Z",
     "start_time": "2020-06-22T01:58:18.970764Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['x', 'xi']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trie.prefixes('xi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:58:23.212928Z",
     "start_time": "2020-06-22T01:58:23.137930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', '1', '10', '2', '3', '4', '5', '6', '7', '8', '9']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trie.suffixes('yong')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:09:06.678072Z",
     "start_time": "2020-06-22T02:09:06.667553Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', '1', '10', '2', '3', '4', '5', '6', '7', '8', '9', 'g']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trie.suffixes('wen')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T15:43:01.672639Z",
     "start_time": "2020-06-21T15:43:01.654997Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abcdefghijklmnopqrstuvwxyz'"
      ]
     },
     "execution_count": 480,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.ascii_lowercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T15:43:02.929213Z",
     "start_time": "2020-06-21T15:43:02.921152Z"
    }
   },
   "outputs": [],
   "source": [
    "a = [1,2,3,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:09:48.527676Z",
     "start_time": "2020-06-22T02:09:48.523684Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'紊'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trie['wen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:10:31.456665Z",
     "start_time": "2020-06-22T02:10:31.437845Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [1,2,3]\n",
    "a[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 文本纠错"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### NOTE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个问题在于，如果文本纠错是一个字一个字替换，然后根据LM打分，那么输入文本较长时，搜索空间指数级增长"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由于在retrival阶段，会按照重要名词进行布尔搜索，所以此处只对名词性词汇进行纠错"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:01:35.907232Z",
     "start_time": "2020-06-22T02:01:35.627198Z"
    }
   },
   "outputs": [],
   "source": [
    "# 加载模型\n",
    "import kenlm\n",
    "\n",
    "# 将文件导入到 kenlm 语言模型中\n",
    "model = kenlm.LanguageModel(\"./model/lm.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:02:15.159519Z",
     "start_time": "2020-06-22T02:02:15.155520Z"
    }
   },
   "outputs": [],
   "source": [
    "from itertools import product"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 增加词频限制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T03:51:27.893286Z",
     "start_time": "2020-06-22T03:51:27.883863Z"
    }
   },
   "outputs": [],
   "source": [
    "from functools import reduce\n",
    "\n",
    "\n",
    "avg = reduce(lambda a, b: a + b, words_frequnece.values()) / len(words_frequnece)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T03:51:33.925532Z",
     "start_time": "2020-06-22T03:51:33.915498Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "576.2143456375838"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T04:08:39.808511Z",
     "start_time": "2020-06-22T04:08:39.778511Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def find_word(string):\n",
    "    pattern = re.compile(r'[\\u4e00-\\u9fa5]|[abcdefghijklmnopqrstuvwxyz]+')\n",
    "    return re.findall(pattern, string.lower())\n",
    "    \n",
    "    \n",
    "def space_split(string):\n",
    "    pre = '@'\n",
    "    line = ''\n",
    "    for s in string:\n",
    "        if ord('A') <= ord(pre) <= ord('z') and  ord('A') <= ord(s) <= ord('z'):\n",
    "            line += s\n",
    "        else:\n",
    "            line += ' ' + s\n",
    "        pre = s\n",
    "    return line\n",
    "    \n",
    "    \n",
    "def check_english(w):\n",
    "    return (ord('a') <= ord(w[0]) <= ord('z')) and ((ord('a') <= ord(w[-1]) <= ord('z')))\n",
    "    \n",
    "\n",
    "def find_candidates(w):\n",
    "    candidates = set()\n",
    "    pin = lazy_pinyin(w)[0]\n",
    "    try:\n",
    "        # prefix查找\n",
    "        not_leaf = list(filter(lambda x: 0 <= (len(pin) - len(x)) <= 1, trie.prefixes(pin)))\n",
    "\n",
    "        if len(not_leaf) == 0:\n",
    "            not_leaf = trie.prefixes(pin)[0]\n",
    "        \n",
    "        for prefix in not_leaf:\n",
    "            suffixes = trie.suffixes(prefix)\n",
    "            for suf in suffixes:\n",
    "                cand = prefix + suf\n",
    "                # 长度限制\n",
    "                if 0 <= len(cand) - len(pin) <= 1:\n",
    "                    candidates.add(trie[cand])\n",
    "                    \n",
    "        candidates.add(w)\n",
    "    except KeyError:\n",
    "        pass\n",
    "    return candidates\n",
    "\n",
    "\n",
    "def frequence_filter(possible_set):\n",
    "    return sorted(list(possible_set), key=lambda w: words_frequnece.get(w, 0), reverse=True)[:5]\n",
    "\n",
    "    \n",
    "def creat_query(self, query, to_create):\n",
    "    start_idx = 0\n",
    "    words_corrected = []\n",
    "    count = 0\n",
    "    for word in to_create:\n",
    "        idx = query.index(word, start_idx)\n",
    "        best_lm_score = -10000\n",
    "        \n",
    "        possible_w = []\n",
    "        if check_english(word):  # 英文直接找\n",
    "            possibles = frequence_filter(find_candidates(word))\n",
    "        else:\n",
    "            for w in word:  # 汉语一个字一个字替换\n",
    "                filter_words = frequence_filter(find_candidates(w))\n",
    "                possible_w.append(filter_words)\n",
    "                \n",
    "            possibles = product(*possible_w)\n",
    "        \n",
    "        candi = ''\n",
    "        \n",
    "        for repl in possibles:\n",
    "#             print(repl)\n",
    "            count += 1\n",
    "            tmp = query[: start_idx] +  query[start_idx: ].replace(word, ''.join(repl), 1)\n",
    "            score = model.score(space_split(tmp))\n",
    "            if score > best_lm_score: \n",
    "                best_lm_score = score\n",
    "                candi = tmp\n",
    "        query = candi\n",
    "        words_corrected.append(query[idx: idx + len(word)])\n",
    "        start_idx = idx + len(word)\n",
    "    print(count)\n",
    "    return query, words_corrected\n",
    "    \n",
    "    \n",
    "#     words = find_word(query)\n",
    "#     return words\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:01:39.523357Z",
     "start_time": "2020-06-22T02:01:39.507358Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-20.38299560546875"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"请 问 怎 么 办 理 新 永 卡\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:01:41.011558Z",
     "start_time": "2020-06-22T02:01:40.994603Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-23.79347038269043"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"请 问 怎 么 办 理 瑕 镛 卡\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T02:01:41.852643Z",
     "start_time": "2020-06-22T02:01:41.843609Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-11.427366256713867"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(\"请 问 怎 么 办 理 信 用 卡\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T04:15:46.614411Z",
     "start_time": "2020-06-22T04:15:46.529381Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1016\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('请吻办visa，需要身份证吗, 请问怎么办了信用卡，你好哈,请吻办visa，需要身份证吗, 请问怎么办了信用卡，你好哈,请吻办visa，需要身份证吗, 请问怎么办了信用卡，你好哈,请吻办visa，需要身份证吗, 请问怎么办了信用卡，你好哈',\n",
       " ['visa',\n",
       "  '身份证',\n",
       "  '信用卡',\n",
       "  'visa',\n",
       "  '身份证',\n",
       "  '信用卡',\n",
       "  'visa',\n",
       "  '身份证',\n",
       "  '信用卡',\n",
       "  'visa',\n",
       "  '身份证',\n",
       "  '信用卡'])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%timeit\n",
    "creat_query(1, \"请吻办vasa，需要深粉证吗, 请问怎么办了新永卡，你好哈,请吻办vasa，需要深粉证吗, 请问怎么办了新永卡，你好哈,请吻办vasa，需要深粉证吗, 请问怎么办了新永卡，你好哈,请吻办vasa，需要深粉证吗, 请问怎么办了新永卡，你好哈\", \n",
    "            [\"vasa\", \"深粉证\", \"新永卡\",\"vasa\", \"深粉证\", \"新永卡\", \"vasa\", \"深粉证\", \"新永卡\", \"vasa\", \"深粉证\", \"新永卡\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T04:46:12.575001Z",
     "start_time": "2020-06-22T04:46:12.553039Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('请问办visa，需要身份证吗, 请问怎么办了下用卡，你好哈',\n",
       " ['请', '问', 'visa', '身份证', '下', '用', '卡', '你', '好'])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "creat_query(1, \"请吻办vasa，需要深粉证吗, 请问怎么办了新永卡，你好哈\", [\"请\",\"吻\", \"vasa\", \"深粉证\", \"新\",\"永\",\"卡\", \"你\", \"好\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:36:14.741998Z",
     "start_time": "2020-06-22T01:36:14.730000Z"
    }
   },
   "outputs": [],
   "source": [
    "from nlp import Ltp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T01:37:23.112181Z",
     "start_time": "2020-06-22T01:37:16.956114Z"
    }
   },
   "outputs": [],
   "source": [
    "ltp = Ltp(pos=True, seg_lexicon_path='data/lexicon_seg.txt', pos_lexicon_path='data/lexicon.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T04:31:25.617282Z",
     "start_time": "2020-06-22T04:31:25.596243Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['请', '吻', '怎么', '伴', '离', '新', '永卡']\n",
      "['v', 'v', 'r', 'v', 'v', 'a', 'n']\n"
     ]
    }
   ],
   "source": [
    "a = ltp.segmentor.segment(\"请吻怎么伴离新永卡\")\n",
    "print(list(a))\n",
    "b = ltp.postagger.postag(a)\n",
    "print(list(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T14:06:06.162561Z",
     "start_time": "2020-06-21T14:06:06.158599Z"
    }
   },
   "outputs": [],
   "source": [
    "to_create = [\"新永卡\", \"银行\", \"vise\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 491,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T16:00:45.968464Z",
     "start_time": "2020-06-21T16:00:45.954469Z"
    }
   },
   "outputs": [],
   "source": [
    "# test\n",
    "# possible_w = []\n",
    "# for word in to_create:\n",
    "#     if check_english(word):\n",
    "#         possible_w.append(find_candidates(word))\n",
    "#     else:\n",
    "#         for w in word:\n",
    "#             possible_w.append(find_candidates(w))\n",
    "\n",
    "# print(possible_w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freq = unpickle_file('data/full_word2id.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "for k, v in word_freq.items():\n",
    "    vocab.add(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "11320"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_pickle('data/vocabulary.bin', vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T15:52:54.353835Z",
     "start_time": "2020-06-21T15:52:54.342808Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils import save_pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-21T15:53:48.187050Z",
     "start_time": "2020-06-21T15:53:48.175034Z"
    }
   },
   "outputs": [],
   "source": [
    "save_pickle('model/create_datrie.bin', trie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyMSUsGOnEK7rk3TxeBW0yhJ",
   "collapsed_sections": [],
   "name": "train_lm.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}